package com.wk.crawler;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.cert.X509Certificate;

public class BasicWebCrawler {

    public static void main(String[] args)throws Exception {

        // 设置忽略SSL验证的信任管理器
        TrustManager[] trustAll = new TrustManager[]{
                new X509TrustManager() {
                    @Override
                    public X509Certificate[] getAcceptedIssuers() {
                        return null;
                    }

                    @Override
                    public void checkClientTrusted(X509Certificate[] certs, String authType) {
                    }

                    @Override
                    public void checkServerTrusted(X509Certificate[] certs, String authType) {
                    }
                }
        };

        SSLContext sslContext = SSLContext.getInstance("TLS");
        sslContext.init(null, trustAll, new java.security.SecureRandom());
        HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory());

        // 禁用主机名验证
        HttpsURLConnection.setDefaultHostnameVerifier((hostname, session) -> true);
        String url = "https://www.macw.cc/talks"; // 替换成你要抓取的网页的URL
        try {
            // 使用Jsoup连接到网页并获取整个HTML文档
            Document doc = Jsoup.connect(url).get();
            
            // 打印网页的标题
            String title = doc.title();
            System.out.println("网页标题: " + title);
            
            // 获取所有的链接
            Elements links = doc.select("a[href]");
            for (Element link : links) {
                // 打印链接的地址和文本
                System.out.println("链接地址: " + link.attr("href"));
                System.out.println("链接文本: " + link.text());
            }
            
            // 获取网页的正文内容
            String text = doc.body().text();
            System.out.println("网页内容: " + text);
            
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}